from os import makedirs

import requests
from bs4 import BeautifulSoup
from contextlib import closing

from os.path import exists


def req_get_bf(url):
    return BeautifulSoup(requests.get(url, timeout=180).text, 'html.parser')


def download(photo_url, dir_name, filename):
    jpg_filename = dir_name + '/' + filename
    if not exists(jpg_filename):
        with closing(requests.get(url=photo_url, stream=True, timeout=90)) as r:
            with open(jpg_filename, 'ab+') as f:
                for chunk in r.iter_content(chunk_size=1024):
                    if chunk:
                        f.write(chunk)
                        f.flush()


def start_down_pic_by_cc(url, detail_name):
    dirName = 'artic.edu'
    if not exists(dirName):
        makedirs(dirName)

    print('下载: {}'.format(detail_name))
    download(url, dirName, detail_name)


def just_url(temp_url, page):
    temp_bf = req_get_bf(temp_url + str(page))
    detail_pic_bf = temp_bf.find_all('a', 'm-listing__link')
    print('================= 第' + str(page) + '页 共' + str(detail_pic_bf.__len__()) + '张' + '==========')

    if detail_pic_bf:
        for i in detail_pic_bf:
            try:
                spanImage = i.find('img')
                if spanImage:
                    imgLinkStr = spanImage['data-srcset']
                    if imgLinkStr:
                        imgLinkSplit = str(imgLinkStr).split(' ')
                        # https://www.artic.edu/iiif/2/a1a694d2-15e5-2464-0f5b-f527f7bfbfd4/full/!3000,3000/0/default.jpg
                        # https://www.artic.edu/iiif/2/a1a694d2-15e5-2464-0f5b-f527f7bfbfd4/full/800,/0/default.jpg
                        imgLinkReal = imgLinkSplit[imgLinkSplit.__len__() - 2].replace("800,", "!3000,3000")
                        detail_name = str(i.find('strong').string) + '.jpg'
                        start_down_pic_by_cc(imgLinkReal, detail_name)

            except requests.exceptions.RequestException:
                pass
            except OSError:
                pass

    just_url(temp_url, page + 1)


if __name__ == '__main__':
    html = "https://www.artic.edu/collection?is_public_domain=1&page="
    just_url(html, 109)
